import time

import scrapy
from scrapy.http import HtmlResponse

from ..items import CbcItem
from selenium import webdriver
from selenium.webdriver import FirefoxOptions
from threading import Thread, Lock
class ChinaSpider(scrapy.Spider):
    name = 'globalnews'
    allowed_domains = ['https://globalnews.ca//']
    start_urls = ['https://globalnews.ca/?s=China&type=news']
    num =0
    count =0
    def __init__(self):
        # 在初始化淘宝对象时，创建driver
        super(ChinaSpider, self).__init__(name='globalnews')
        option = FirefoxOptions()
        option.headless = True
        self.driver = webdriver.Firefox(options=option)


    def parse(self, response):
        if self.count == 0:
            print("我只运行了一次")
            self.count=1
            self.driver.get(response.url)
            for i in range(20):
                button = self.driver.find_element_by_xpath(
                    "//div[@id='search-results']/section/button[@id='load-more-results']")
                self.driver.execute_script("arguments[0].click();", button)
                print("我已经点击了%d" % i)
                time.sleep(3)
            response = HtmlResponse(url=response.url, encoding='utf8', body=self.driver.page_source)

        tr_list = response.xpath("//div[@class='story']")
        print("数据条数"+str(len(tr_list)))
        for tr in tr_list:
            item = CbcItem()
            href = tr.xpath("./article/div/h3/a/@href").extract_first()
            item['herf'] = href
            item["title"] = tr.xpath("normalize-space(./article/div/h3/a/text())").extract_first()
            item["brief"] = tr.xpath(
                "normalize-space(./article/div/div[@class='story-txt']/text())").extract_first()

            yield scrapy.Request(
                item['herf'],
                callback=self.parse_detail,
                meta={"item": item},
                dont_filter=True
            )
    def parse_detail(self, response):
        item = response.meta['item']
        item["content"] = response.xpath("//*[@id='content']/div[1]/div[1]/div[3]/article/p/text()").extract()
        item["content"] = "".join(item["content"])
        time = response.xpath(
            "//*[@id='article-byline']/div[2]/div[2]/div/div[1]/span/text()").extract_first()
        if time is None:
            time = response.xpath("//*[@id='article-byline']/div/div[2]/div/div[1]/span").extract_first()
        time = time.split(" ")
        item["publish_date"] = time[1]
        print("我爬了___"+str(self.num))
        self.num = self.num+1
        yield (item)